{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c4b172d-b6f2-4e9d-b300-495666e5b4a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, Markdown\n",
    "import pandas as pd \n",
    "import numpy as np \n",
    "import ast \n",
    "import json\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "import matplotlib.transforms as transforms\n",
    "import seaborn as sns \n",
    "\n",
    "import statsmodels.formula.api as smf\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import re \n",
    "from tqdm import tqdm \n",
    "import sys "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "188284b1-ba0c-46f5-91b5-c56662338a89",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eaada50b-89c1-4439-a9e6-c2ae4942fcb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_coeffs(model, interactions_only=False, ax=None, label=None):\n",
    "    \n",
    "    dict_iv_name = {\"C(treatment_social, Treatment(reference='Control'))[T.Chat]\":\"Chat\", \n",
    "                    \"C(treatment_social, Treatment(reference='Control'))[T.Indicators]\":\"Indicators\",\n",
    "                    \"C(treatment_explanation, Treatment(reference='None'))[T.Local]\":\"Explanations\",\n",
    "                    \"treatment_feedback[T.True]\":\"Feedback\",\n",
    "                    \"C(treatment_social, Treatment(reference='Control'))[T.Chat]:C(treatment_explanation, Treatment(reference='None'))[T.Local]\":\"Chat:Explanations\",\n",
    "                    \"C(treatment_social, Treatment(reference='Control'))[T.Indicators]:C(treatment_explanation, Treatment(reference='None'))[T.Local]\":\"Indicators:Explanations\",\n",
    "                    \"C(treatment_social, Treatment(reference='Control'))[T.Chat]:treatment_feedback[T.True]\":\"Chat:Feedback\",\n",
    "                    \"C(treatment_social, Treatment(reference='Control'))[T.Indicators]:treatment_feedback[T.True]\":\"Indicators:Feedback\"}\n",
    "    \n",
    "    plot_df = pd.DataFrame([model.params, model.bse]).T.reset_index().rename(columns={\"index\":\"iv\", 0:\"coef\", 1:\"se\"}).query(\"iv != 'Intercept' and iv != 'playerId Var'\").assign(iv = lambda x: x.iv.map(dict_iv_name)).dropna(subset=\"iv\")\n",
    "    \n",
    "    # if interactions_only:\n",
    "    #     plot_df = plot_df[[\":\" in x for x in plot_df[\"iv\"]]].sort_values(\"coef\")\n",
    "    # else:\n",
    "    #     plot_df = plot_df.sort_values(\"coef\")\n",
    "    \n",
    "    # plot_df = plot_df.sort_values(\"coef\")\n",
    "    sorter = ['Indicators:Explanations','Indicators:Feedback','Chat:Explanations','Chat:Feedback','Indicators','Chat','Explanations','Feedback']\n",
    "    plot_df.sort_values(by=\"iv\", key=lambda column: column.map(lambda e: sorter.index(e)), inplace=True)\n",
    "    \n",
    "    if ax:\n",
    "        ax.scatter(x=plot_df[\"coef\"], y=plot_df[\"iv\"], label=label, s=60)\n",
    "        ax.errorbar(x=plot_df[\"coef\"], y=plot_df[\"iv\"], xerr=1.96*plot_df[\"se\"], ls=\"none\", linewidth=2)\n",
    "        ax.axvline(x=0, color=\"black\", linestyle=\"--\")\n",
    "    else:\n",
    "        plt.scatter(x=plot_df[\"coef\"], y=plot_df[\"iv\"])\n",
    "        plt.errorbar(x=plot_df[\"coef\"], y=plot_df[\"iv\"], xerr=1.96*plot_df[\"se\"], ls=\"none\")\n",
    "        plt.axvline(x=0, color=\"black\", linestyle=\"--\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df528f03-c04a-419f-8489-4e2574fbd238",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_robustness_checks(filter_string):\n",
    "    \n",
    "    df_temp = df_analysis_dropout.query(filter_string, engine=\"python\")\n",
    "    \n",
    "    for couple_id in df_temp[\"couple_id\"].unique():\n",
    "        df_temp.loc[df_temp[\"couple_id\"] == couple_id, [\"zscore_woa\", \"zscore_dq\"]] = StandardScaler().fit_transform(df_temp.loc[df_temp[\"couple_id\"] == couple_id, [\"woa\", \"dq\"]])\n",
    "    \n",
    "    \n",
    "    model_woa = smf.mixedlm(\"zscore_woa ~ C(treatment_social, Treatment(reference='Control')) * (C(treatment_explanation, Treatment(reference='None')) + treatment_feedback)\", \n",
    "                            data=df_temp, groups=\"playerId\").fit()\n",
    "\n",
    "    model_dq = smf.mixedlm(\"zscore_dq ~ C(treatment_social, Treatment(reference='Control')) * (C(treatment_explanation, Treatment(reference='None')) + treatment_feedback)\", \n",
    "                            data=df_temp, groups=\"playerId\").fit()\n",
    "\n",
    "    fig, axes = plt.subplots(nrows=1, ncols=2, sharey=True, sharex=True, figsize=(10,8))\n",
    "\n",
    "    plot_coeffs(model_woa, ax=axes[0])\n",
    "    plot_coeffs(model_dq, ax=axes[1])\n",
    "    \n",
    "    axes[0].tick_params(labelsize=15)\n",
    "    axes[1].tick_params(labelsize=15)\n",
    "\n",
    "\n",
    "    axes[0].set_xlim(-0.5, 0.5)\n",
    "    axes[1].set_xlim(-0.5, 0.5)\n",
    "\n",
    "    axes[0].set_title(\"Weight of Advice\", fontsize=15)\n",
    "    axes[1].set_title(\"Delegation Quality\", fontsize=15)\n",
    "\n",
    "    axes[0].set_xlabel(\"Coef.\")\n",
    "    axes[1].set_xlabel(\"Coef.\")\n",
    "\n",
    "    plt.show()\n",
    "    \n",
    "    return model_woa, model_dq "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ae89f58-b938-4bc2-ba72-5acba871fb38",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Data pre-processing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbe28680-fed7-4869-8933-bdeaa3644f75",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_factors = pd.DataFrame([{\"treatment_name\":\"A1\", \"treatment_social\":\"Control\", \"treatment_explanation\":\"None\", \"treatment_feedback\":False},\n",
    "                           {\"treatment_name\":\"A2\", \"treatment_social\":\"Control\", \"treatment_explanation\":\"None\", \"treatment_feedback\":True},\n",
    "                           {\"treatment_name\":\"A3\", \"treatment_social\":\"Control\", \"treatment_explanation\":\"Local\", \"treatment_feedback\":False},\n",
    "                           {\"treatment_name\":\"A4\", \"treatment_social\":\"Control\", \"treatment_explanation\":\"Local\", \"treatment_feedback\":True}, \n",
    "                           {\"treatment_name\":\"B1\", \"treatment_social\":\"Chat\", \"treatment_explanation\":\"None\", \"treatment_feedback\":False},\n",
    "                           {\"treatment_name\":\"B2\", \"treatment_social\":\"Chat\", \"treatment_explanation\":\"None\", \"treatment_feedback\":True},\n",
    "                           {\"treatment_name\":\"B3\", \"treatment_social\":\"Chat\", \"treatment_explanation\":\"Local\", \"treatment_feedback\":False},\n",
    "                           {\"treatment_name\":\"B4\", \"treatment_social\":\"Chat\", \"treatment_explanation\":\"Local\", \"treatment_feedback\":True},\n",
    "                           {\"treatment_name\":\"C1\", \"treatment_social\":\"Indicators\", \"treatment_explanation\":\"None\", \"treatment_feedback\":False},\n",
    "                           {\"treatment_name\":\"C2\", \"treatment_social\":\"Indicators\", \"treatment_explanation\":\"None\", \"treatment_feedback\":True},\n",
    "                           {\"treatment_name\":\"C3\", \"treatment_social\":\"Indicators\", \"treatment_explanation\":\"Local\", \"treatment_feedback\":False},\n",
    "                           {\"treatment_name\":\"C4\", \"treatment_social\":\"Indicators\", \"treatment_explanation\":\"Local\", \"treatment_feedback\":True}])\n",
    "\n",
    "dict_info_types = {\"1\":\"NoFeedback_NoExpl\", \"2\":\"FeedbackOnly\", \"3\":\"ExplOnly\", \"4\":\"Feedback+Expl\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d88d71d-b11f-4615-8b14-39acff1b88e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath = \"../data\"\n",
    "\n",
    "# Load basic dataframes\n",
    "df_players = pd.read_csv(filepath+\"/players.csv\")\n",
    "df_games = pd.read_csv(filepath+\"/games.csv\")\n",
    "df_rounds = pd.read_csv(filepath+\"/player-rounds.csv\").rename(columns={\"data.previousPrediction\":\"initial_pred\", \"data.prediction\":\"final_pred\", \"data.score\":\"score\"})\n",
    "df_stages = pd.read_csv(filepath+\"/player-stages.csv\")\n",
    "df_rounds_meta = pd.read_csv(filepath+\"/rounds.csv\").rename(columns={\"data.chat\":\"chat_log\", \"data.effectiveIndex\":\"effectiveIndex\", \"data.case\":\"round_type\"}).sort_values(\"effectiveIndex\")\n",
    "df_rounds_meta.loc[~df_rounds_meta[\"chat_log\"].isnull(), \"parsed_chat_log\"] = [ast.literal_eval(x) for x in df_rounds_meta.loc[~df_rounds_meta[\"chat_log\"].isnull(), \"chat_log\"]]\n",
    "df_treatments = pd.read_csv(filepath+\"/treatments.csv\")\n",
    "\n",
    "# Parse task instance data \n",
    "df_rounds[\"task_data\"] = df_rounds[\"roundId\"].map(dict(zip(df_rounds_meta[\"_id\"], df_rounds_meta[\"data.task\"])))\n",
    "df_rounds[\"round_type\"] = df_rounds[\"roundId\"].map(dict(zip(df_rounds_meta[\"_id\"], df_rounds_meta[\"round_type\"])))\n",
    "df_rounds[\"task_index\"] = df_rounds[\"roundId\"].map(dict(zip(df_rounds_meta[\"_id\"], df_rounds_meta[\"effectiveIndex\"])))\n",
    "\n",
    "\n",
    "# Parse human and model predictions \n",
    "df_analysis = df_rounds.query(\"round_type == 'revise'\").reset_index(drop=True).assign(task_data = lambda x: [ast.literal_eval(y) for y in x.task_data])\n",
    "df_analysis[\"model_pred\"] = [x[\"model_prediction_prob\"] for x in df_analysis[\"task_data\"]]\n",
    "df_analysis[\"model_answer\"] = df_analysis[\"model_pred\"].round()\n",
    "df_analysis[\"couple_id\"] = [x[\"_id\"] for x in df_analysis[\"task_data\"]]\n",
    "df_analysis[\"correct_answer\"] = [{\"Yes\":1, \"No\":0}[x[\"correct_answer\"]] for x in df_analysis[\"task_data\"]]\n",
    "df_analysis[\"model_correct\"] = df_analysis[\"model_pred\"].round() == df_analysis[\"correct_answer\"]\n",
    "df_analysis[\"final_pred_is_binary\"] = (df_analysis[\"final_pred\"] == 1) | (df_analysis[\"final_pred\"] == 0)\n",
    "df_analysis[\"initial_pred_is_binary\"] = (df_analysis[\"initial_pred\"] == 1) | (df_analysis[\"initial_pred\"] == 0)\n",
    "\n",
    "\n",
    "# Get treatment info \n",
    "df_game_treatments = (df_games[[\"_id\", \"treatmentId\"]].rename(columns={\"_id\":\"gameId\"})\n",
    "                      .merge(df_treatments[[\"_id\", \"name\"]].rename(columns={\"name\":\"treatment_name\"}), \n",
    "                             left_on=\"treatmentId\", right_on=\"_id\", how=\"left\")).merge(df_factors, on=\"treatment_name\", how=\"left\")\n",
    "\n",
    "# Calculate outcomes (WoA and DQ)\n",
    "df_analysis = df_analysis.merge(df_game_treatments.filter(regex=\"gameId|treatment_\"), on=\"gameId\", how=\"left\")\n",
    "df_analysis[\"treatment_class\"] = df_analysis[\"treatment_name\"].str.slice(1,2).map(dict_info_types).astype(\"category\").cat.reorder_categories(new_categories=[\"NoFeedback_NoExpl\",\"FeedbackOnly\",\"ExplOnly\",\"Feedback+Expl\"], ordered=True)\n",
    "df_analysis[\"initial_model_delta\"] = abs(df_analysis[\"initial_pred\"] - df_analysis[\"model_pred\"]) > 0.1\n",
    "df_analysis[\"woa\"] = (df_analysis[\"final_pred\"] - df_analysis[\"initial_pred\"]) / (df_analysis[\"model_pred\"] - df_analysis[\"initial_pred\"])\n",
    "df_analysis[\"error_initial\"] = abs(df_analysis[\"initial_pred\"] - df_analysis[\"correct_answer\"])\n",
    "df_analysis[\"error_final\"] = abs(df_analysis[\"final_pred\"] - df_analysis[\"correct_answer\"])\n",
    "df_analysis[\"error_model\"] = abs(df_analysis[\"model_pred\"] - df_analysis[\"correct_answer\"])\n",
    "df_analysis[\"dq_alpha\"] = df_analysis[\"error_initial\"] - df_analysis[\"error_model\"]\n",
    "df_analysis[\"dq\"] = df_analysis[\"dq_alpha\"] * df_analysis[\"woa\"]\n",
    "df_analysis[\"m1_error\"] = df_analysis[\"error_final\"] / (df_analysis[[\"error_initial\", \"error_model\"]].min(axis=1) + 0.001)\n",
    "\n",
    "# Observed accuracy across all previous rounds for the current round \n",
    "df_observed_acc = df_analysis.drop_duplicates(subset=[\"gameId\", \"task_index\"]).filter(regex=\"gameId|index|rect\").reset_index()\n",
    "df_observed_acc[\"observed_acc\"] = df_observed_acc.groupby(\"gameId\")[\"model_correct\"].transform(lambda x: x.shift().rolling(10,1).mean())\n",
    "df_analysis = df_analysis.merge(df_observed_acc[[\"gameId\", \"task_index\", \"observed_acc\"]], on=[\"gameId\", \"task_index\"], how=\"left\")\n",
    "df_analysis[\"previous_model_correct\"] = df_analysis.groupby(\"playerId\")[\"model_correct\"].shift()\n",
    "\n",
    "# Get chat stats \n",
    "df_chat_analysis = (df_rounds_meta.dropna(subset=[\"parsed_chat_log\"])\n",
    "                    .assign(n_messages = lambda x: [len(y) if type(y) == tuple else 1 for y in x.parsed_chat_log]))\n",
    "\n",
    "dict_game_total_messages = dict(df_chat_analysis.groupby(\"gameId\")[\"n_messages\"].sum())\n",
    "dict_round_messages = dict(df_chat_analysis.groupby(\"_id\")[\"n_messages\"].sum())\n",
    "\n",
    "df_analysis[\"n_game_messages\"] = df_analysis[\"gameId\"].map(dict_game_total_messages).fillna(0)\n",
    "df_analysis = df_analysis.merge(df_chat_analysis.rename(columns={\"effectiveIndex\":\"task_index\", \"n_messages\":\"n_round_messages\"})[[\"gameId\", \"task_index\", \"n_round_messages\"]],\n",
    "                  on=[\"gameId\", \"task_index\"], how=\"left\")\n",
    "\n",
    "\n",
    "# Get peer outcomes for indicator analysis \n",
    "df_peer_outcomes = df_analysis.groupby([\"gameId\", \"task_index\"]).apply(lambda x: dict(zip(x[\"playerId\"], x[\"woa\"]))).reset_index().rename(columns={0:\"group_woa\"}).merge(\n",
    "df_analysis.groupby([\"gameId\", \"task_index\"]).apply(lambda x: dict(zip(x[\"playerId\"], x[\"score\"]))).reset_index().rename(columns={0:\"group_score\"}), on=[\"gameId\", \"task_index\"], how=\"left\")\n",
    "\n",
    "df_analysis = df_analysis.merge(df_peer_outcomes, on=[\"gameId\", \"task_index\"], how=\"left\")\n",
    "df_analysis[\"peer_keys\"] = df_analysis.apply(lambda x: [k for k,v in x[\"group_woa\"].items() if k != x[\"playerId\"]], axis=1) \n",
    "df_analysis[\"peer_woa\"] = df_analysis.apply(lambda x: [x[\"group_woa\"][key] for key in x[\"peer_keys\"]], axis=1)\n",
    "df_analysis[\"peer_score\"] = df_analysis.apply(lambda x: [x[\"group_score\"][key] for key in x[\"peer_keys\"]], axis=1)\n",
    "# df_analysis[\"peer_score\"] = df_analysis.apply(lambda x: [v for k,v in x[\"group_score\"].items() if k != x[\"playerId\"]], axis=1)\n",
    "df_analysis[\"previous_peer_woa\"] = df_analysis.groupby(\"playerId\")[\"peer_woa\"].shift()\n",
    "df_analysis[\"previous_peer_score\"] = df_analysis.groupby(\"playerId\")[\"peer_score\"].shift()\n",
    "df_analysis[\"mean_peer_woa\"] = [np.nanmean(x) for x in df_analysis[\"peer_woa\"]]\n",
    "df_analysis[\"mean_peer_score\"] = [np.nanmean(x) for x in df_analysis[\"peer_score\"]]\n",
    "df_analysis[\"previous_mean_peer_woa\"] = df_analysis.groupby(\"playerId\")[\"mean_peer_woa\"].shift()\n",
    "df_analysis[\"previous_mean_peer_score\"] = df_analysis.groupby(\"playerId\")[\"mean_peer_score\"].shift()\n",
    "df_analysis[\"previous_woa\"] = df_analysis.groupby(\"playerId\")[\"woa\"].shift()\n",
    "df_analysis[\"previous_score\"] = df_analysis.groupby(\"playerId\")[\"score\"].shift()\n",
    "df_analysis[\"previous_woa_above_peer_mean\"] = df_analysis[\"previous_woa\"] > df_analysis[\"previous_mean_peer_woa\"]\n",
    "df_analysis[\"previous_score_above_peer_mean\"] = df_analysis[\"previous_score\"] > df_analysis[\"previous_mean_peer_score\"]\n",
    "df_analysis[\"woa_change\"] = df_analysis[\"woa\"] - df_analysis[\"previous_woa\"]\n",
    "df_analysis[\"woa_increase\"] = df_analysis[\"woa\"] > df_analysis[\"previous_woa\"]\n",
    "df_analysis[\"compare_woa_above_peers\"] = df_analysis.apply(lambda x: [(x[\"woa\"] - y) > 0.1 for y in x[\"peer_woa\"]], axis=1)\n",
    "df_analysis[\"compare_score_above_peers\"] = df_analysis.apply(lambda x: [x[\"score\"] > y for y in x[\"peer_score\"]], axis=1)\n",
    "df_analysis[\"signals_to_rely_more\"] = df_analysis.apply(lambda x: sum(np.array(x[\"compare_woa_above_peers\"]) == np.array(x[\"compare_score_above_peers\"])), axis=1)\n",
    "df_analysis[\"previous_signals_to_rely_more\"] = df_analysis.groupby(\"playerId\")[\"signals_to_rely_more\"].shift()\n",
    "df_analysis[\"num_peers_relied_more\"] = df_analysis.apply(lambda x: 2-sum(np.array(x[\"compare_woa_above_peers\"])), axis=1)\n",
    "df_analysis[\"previous_num_peers_relied_more\"] = df_analysis.groupby(\"playerId\")[\"num_peers_relied_more\"].shift()\n",
    "df_analysis[\"num_peers_scored_more\"] = df_analysis.apply(lambda x: 2-sum(np.array(x[\"compare_score_above_peers\"])), axis=1)\n",
    "df_analysis[\"previous_num_peers_scored_more\"] = df_analysis.groupby(\"playerId\")[\"num_peers_scored_more\"].shift()\n",
    "\n",
    "df_analysis.loc[(df_analysis[\"previous_woa_above_peer_mean\"]) & (df_analysis[\"previous_score_above_peer_mean\"]), \"indicator_perception\"] = \"Others rely less, score less\"\n",
    "df_analysis.loc[(df_analysis[\"previous_woa_above_peer_mean\"]) & (~df_analysis[\"previous_score_above_peer_mean\"]), \"indicator_perception\"] = \"Others rely more, score more\"\n",
    "df_analysis.loc[(~df_analysis[\"previous_woa_above_peer_mean\"]) & (df_analysis[\"previous_score_above_peer_mean\"]), \"indicator_perception\"] = \"Rely less, score higher\"\n",
    "df_analysis.loc[(~df_analysis[\"previous_woa_above_peer_mean\"]) & (~df_analysis[\"previous_score_above_peer_mean\"]), \"indicator_perception\"] = \"Rely less, score lower\"\n",
    "\n",
    "df_analysis[\"zscore_n_game_messages\"] = StandardScaler().fit(np.array(df_analysis.drop_duplicates(\"gameId\")[\"n_game_messages\"]).reshape(-1, 1)).transform(np.array(df_analysis[\"n_game_messages\"]).reshape(-1, 1))\n",
    "df_analysis[\"zscore_n_round_messages\"] = StandardScaler().fit_transform(np.array(df_analysis[\"n_round_messages\"]).reshape(-1, 1))\n",
    "\n",
    "set_dropout_affected_games = set(df_analysis.query(\"initial_pred.isnull()\", engine=\"python\")[\"gameId\"])\n",
    "df_analysis_dropout = df_analysis.query(\"gameId not in @set_dropout_affected_games\").reset_index(drop=True)\n",
    "df_final = df_analysis.query(\"gameId not in @set_dropout_affected_games and initial_model_delta\")\n",
    "\n",
    "\n",
    "# z-score outcomes within task \n",
    "for couple_id in df_final[\"couple_id\"].unique():\n",
    "    df_final.loc[df_final[\"couple_id\"] == couple_id, [\"zscore_woa\", \"zscore_dq\", \"zscore_m1\"]] = StandardScaler().fit_transform(df_final.loc[df_final[\"couple_id\"] == couple_id, [\"woa\", \"dq\", \"m1_error\"]])\n",
    "    \n",
    "    \n",
    "for couple_id in df_analysis_dropout[\"couple_id\"].unique():\n",
    "    df_analysis_dropout.loc[df_analysis_dropout[\"couple_id\"] == couple_id, [\"zscore_woa\", \"zscore_dq\"]] = StandardScaler().fit_transform(df_analysis_dropout.loc[df_analysis_dropout[\"couple_id\"] == couple_id, [\"woa\", \"dq\"]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be95d23b-0fbd-4a8f-bc9b-478e72091ffb",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Figure 3: Main analysis \n",
    "* Includes regressions for Tables S1-S8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8000e20-1bc5-4a4b-b922-672b85afaaf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_woa = smf.mixedlm(\"zscore_woa ~ C(treatment_social, Treatment(reference='Control')) * (C(treatment_explanation, Treatment(reference='None')) + treatment_feedback)\", \n",
    "                        data=df_final, groups=\"playerId\").fit()\n",
    "\n",
    "model_dq = smf.mixedlm(\"zscore_dq ~ C(treatment_social, Treatment(reference='Control')) * (C(treatment_explanation, Treatment(reference='None')) + treatment_feedback)\", \n",
    "                        data=df_final, groups=\"playerId\").fit()\n",
    "\n",
    "# display(model_woa.summary())\n",
    "# display(model_dq.summary())\n",
    "\n",
    "fig, axes = plt.subplots(nrows=1, ncols=2, sharey=True, sharex=True, figsize=(10,8))\n",
    "\n",
    "plot_coeffs(model_woa, ax=axes[0])\n",
    "plot_coeffs(model_dq, ax=axes[1])\n",
    "\n",
    "axes[0].tick_params(labelsize=15)\n",
    "axes[1].tick_params(labelsize=15)\n",
    "axes[0].set_xlim(-0.5, 0.5)\n",
    "axes[1].set_xlim(-0.5, 0.5)\n",
    "axes[0].set_title(\"Weight of Advice\", fontsize=20)\n",
    "axes[1].set_title(\"Delegation Quality\", fontsize=20)\n",
    "axes[0].set_xlabel(\"Coef.\", fontsize=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d29a29e-4bf1-4449-add9-ccae778fa735",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_woa.summary(xname_fe=[\"Intercept\", \"Chat\", \"Indicators\", \"Explanations\", \"Feedback\", \"Chat:Explanation\", \"Indicators:Explanation\", \"Chat:Feedback\", \"Indicators:Feedback\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a199aa3e-e285-49ae-b022-7ebac4637604",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dq.summary(xname_fe=[\"Intercept\", \"Chat\", \"Indicators\", \"Explanations\", \"Feedback\", \"Chat:Explanation\", \"Indicators:Explanation\", \"Chat:Feedback\", \"Indicators:Feedback\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd7cc379-79cb-4c49-8835-6fbc9c33f4a2",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Figure 3 Robustness Check: Threshold = 0.15 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53662a11-fbd3-4947-a359-4f52da7bf7aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "rcheck_015_woa, rcheck_015_dq = filter_robustness_checks(\"abs(initial_pred - model_pred) > 0.15\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "114a839d-0782-452a-b10b-57e995aed52d",
   "metadata": {},
   "outputs": [],
   "source": [
    "rcheck_015_woa.summary(xname_fe=[\"Intercept\", \"Chat\", \"Indicators\", \"Explanations\", \"Feedback\", \"Chat:Explanation\", \"Indicators:Explanation\", \"Chat:Feedback\", \"Indicators:Feedback\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "403b6db2-0ccb-4d5f-b412-399859c1596b",
   "metadata": {},
   "outputs": [],
   "source": [
    "rcheck_015_dq.summary(xname_fe=[\"Intercept\", \"Chat\", \"Indicators\", \"Explanations\", \"Feedback\", \"Chat:Explanation\", \"Indicators:Explanation\", \"Chat:Feedback\", \"Indicators:Feedback\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7f045f2-4553-4bb6-a0d0-53ec692c73c7",
   "metadata": {},
   "source": [
    "## Figure 3 Robustness Check: Threshold = 0.05 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40e478fb-1ddb-48fe-84ab-f14c6b1e6fc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "rcheck_005_woa, rcheck_005_dq = filter_robustness_checks(\"abs(initial_pred - model_pred) > 0.05\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae9fbee9-761e-4013-97a8-d60bc8543699",
   "metadata": {},
   "outputs": [],
   "source": [
    "rcheck_005_woa.summary(xname_fe=[\"Intercept\", \"Chat\", \"Indicators\", \"Explanations\", \"Feedback\", \"Chat:Explanation\", \"Indicators:Explanation\", \"Chat:Feedback\", \"Indicators:Feedback\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c19d6959-7d87-4f15-bada-d87720ad8aaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "rcheck_005_dq.summary(xname_fe=[\"Intercept\", \"Chat\", \"Indicators\", \"Explanations\", \"Feedback\", \"Chat:Explanation\", \"Indicators:Explanation\", \"Chat:Feedback\", \"Indicators:Feedback\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d1547e0-e020-4c4d-938e-6141342fde78",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Figure 3 Robustness Check: Participant dropout filtering  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd51e3fb-8534-4938-af54-b0331267183f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_analysis_dropout_robustness = df_analysis.copy()\n",
    "df_analysis_dropout_robustness[\"first_dropout_round_index\"] = df_analysis_dropout_robustness[\"gameId\"].map(dict(df_analysis.query(\"initial_pred.isnull()\", engine=\"python\").groupby(\"gameId\")[\"task_index\"].min()[:100])).fillna(11)\n",
    "df_analysis_dropout_robustness = df_analysis_dropout_robustness.query(\"task_index < first_dropout_round_index\").reset_index(drop=True)\n",
    "\n",
    "for couple_id in df_analysis_dropout_robustness[\"couple_id\"].unique():\n",
    "    df_analysis_dropout_robustness.loc[df_analysis_dropout_robustness[\"couple_id\"] == couple_id, [\"zscore_woa\", \"zscore_dq\", \"zscore_m1\"]] = StandardScaler().fit_transform(df_analysis_dropout_robustness.loc[df_analysis_dropout_robustness[\"couple_id\"] == couple_id, [\"woa\", \"dq\", \"m1_error\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b42b9617-3da4-4b67-8c97-229d5cb32b74",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_analysis_dropout_robustness.query(\"task_index < first_dropout_round_index\").groupby('treatment_name')['gameId'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7fb509b-09a3-4b95-84b3-4a29149bd337",
   "metadata": {},
   "outputs": [],
   "source": [
    "rcheck_dropout_woa = smf.mixedlm(\"zscore_woa ~ C(treatment_social, Treatment(reference='Control')) * (C(treatment_explanation, Treatment(reference='None')) + treatment_feedback)\", \n",
    "                        data=df_analysis_dropout_robustness.query(\"task_index < first_dropout_round_index\"), groups=\"playerId\").fit()\n",
    "\n",
    "rcheck_dropout_dq = smf.mixedlm(\"zscore_dq ~ C(treatment_social, Treatment(reference='Control')) * (C(treatment_explanation, Treatment(reference='None')) + treatment_feedback)\", \n",
    "                        data=df_analysis_dropout_robustness.query(\"task_index < first_dropout_round_index\"), groups=\"playerId\").fit()\n",
    "\n",
    "\n",
    "fig, axes = plt.subplots(nrows=1, ncols=2, sharey=True, sharex=True, figsize=(10,8))\n",
    "\n",
    "plot_coeffs(rcheck_dropout_woa, ax=axes[0])\n",
    "plot_coeffs(rcheck_dropout_dq, ax=axes[1])\n",
    "\n",
    "axes[0].tick_params(labelsize=15)\n",
    "axes[1].tick_params(labelsize=15)\n",
    "\n",
    "\n",
    "axes[0].set_xlim(-0.5, 0.5)\n",
    "axes[1].set_xlim(-0.5, 0.5)\n",
    "\n",
    "axes[0].set_title(\"Weight of Advice\", fontsize=15)\n",
    "axes[1].set_title(\"Delegation Quality\", fontsize=15)\n",
    "\n",
    "axes[0].set_xlabel(\"Coef.\")\n",
    "axes[1].set_xlabel(\"Coef.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1bdb121-3ebc-4ac0-8fa8-d35445e63c25",
   "metadata": {},
   "outputs": [],
   "source": [
    "rcheck_dropout_woa.summary(xname_fe=[\"Intercept\", \"Chat\", \"Indicators\", \"Explanations\", \"Feedback\", \"Chat:Explanation\", \"Indicators:Explanation\", \"Chat:Feedback\", \"Indicators:Feedback\"]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4b867e4-44e1-47a6-a48a-da82441d4302",
   "metadata": {},
   "outputs": [],
   "source": [
    "rcheck_dropout_dq.summary(xname_fe=[\"Intercept\", \"Chat\", \"Indicators\", \"Explanations\", \"Feedback\", \"Chat:Explanation\", \"Indicators:Explanation\", \"Chat:Feedback\", \"Indicators:Feedback\"]) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4b47a6a-2e2e-45a4-be91-aba15d79acdb",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Figure 4: Frequency of positive and negative peer signals through chat \n",
    "* Includes regression of WoA on positive/negative peer signals (Table S9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f918d646-a460-4016-83e2-32247bf92296",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_manual_ratings = pd.read_csv(\"../data/df_manual_labels_sentiment.csv\").iloc[:,1:]\n",
    "df_manual_ratings.columns = list(df_manual_ratings.columns)[:-1] + [\"notes\"]\n",
    "df_manual_ratings[\"label_encourage\"] = df_manual_ratings[\"label_encourage\"].fillna(0)\n",
    "df_manual_ratings.loc[df_manual_ratings[\"label_encourage\"] == 2, \"label_encourage\"] = -1\n",
    "df_manual_ratings = df_manual_ratings.merge(df_analysis[['treatment_name', 'treatment_social', 'treatment_explanation', 'treatment_feedback', 'treatment_class', \"gameId\", \"task_index\"]].drop_duplicates(), \n",
    "                         on=[\"gameId\", \"task_index\"], how=\"left\")\n",
    "\n",
    "df_manual_ratings[\"positive_message\"] = df_manual_ratings[\"label_encourage\"] == 1\n",
    "df_manual_ratings[\"negative_message\"] = df_manual_ratings[\"label_encourage\"] == -1\n",
    "\n",
    "df_experienced_messages = (df_manual_ratings.groupby([\"gameId\", \"playerId\", \"task_index\"])[[\"positive_message\", \"negative_message\"]].sum().reset_index()\n",
    "                          .merge(df_manual_ratings.groupby([\"gameId\", \"task_index\"])[[\"positive_message\", \"negative_message\"]].sum().reset_index().rename(columns={\"positive_message\":\"group_round_pos_messages\", \n",
    "                                                                                                                                  \"negative_message\":\"group_round_neg_messages\"}),\n",
    "                                 on=[\"gameId\", \"task_index\"], how=\"left\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6d1a9c2-6404-4948-8027-8a21a55c7fae",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_analysis_message_info = df_analysis_dropout.query(\"treatment_social == 'Chat'\").merge(df_experienced_messages, on=[\"gameId\", \"playerId\", \"task_index\"], how=\"left\")\n",
    "df_analysis_message_info[\"positive_message\"] = df_analysis_message_info[\"positive_message\"].fillna(0)\n",
    "df_analysis_message_info[\"negative_message\"] = df_analysis_message_info[\"negative_message\"].fillna(0)\n",
    "df_analysis_message_info[\"positive_message\"] = df_analysis_message_info[\"positive_message\"].fillna(0)\n",
    "df_analysis_message_info[\"group_round_pos_messages\"] = df_analysis_message_info[\"group_round_pos_messages\"].fillna(0)\n",
    "df_analysis_message_info[\"group_round_neg_messages\"] = df_analysis_message_info[\"group_round_neg_messages\"].fillna(0)\n",
    "df_analysis_message_info[\"n_round_messages\"] = df_analysis_message_info[\"n_round_messages\"].fillna(0)\n",
    "\n",
    "\n",
    "df_analysis_message_info[\"experienced_round_pos_messages\"] = df_analysis_message_info[\"group_round_pos_messages\"] - df_analysis_message_info[\"positive_message\"]\n",
    "df_analysis_message_info[\"experienced_round_neg_messages\"] = df_analysis_message_info[\"group_round_neg_messages\"] - df_analysis_message_info[\"negative_message\"]\n",
    "\n",
    "df_analysis_message_info[\"previous_exp_pos_msg\"] = df_analysis_message_info.groupby(\"playerId\")[\"experienced_round_pos_messages\"].shift()\n",
    "df_analysis_message_info[\"previous_exp_neg_msg\"] = df_analysis_message_info.groupby(\"playerId\")[\"experienced_round_neg_messages\"].shift()\n",
    "df_analysis_message_info[\"previous_n_round_messages\"] = df_analysis_message_info.groupby(\"playerId\")[\"n_round_messages\"].shift()\n",
    "\n",
    "df_analysis_message_info = df_analysis_message_info.query(\"gameId not in @set_dropout_affected_games and initial_model_delta\").reset_index()\n",
    "\n",
    "for couple_id in df_analysis_message_info[\"couple_id\"].unique():\n",
    "    df_analysis_message_info.loc[df_analysis_message_info[\"couple_id\"] == couple_id, [\"zscore_woa\", \"zscore_dq\", \"zscore_m1\"]] = StandardScaler().fit_transform(df_analysis_message_info.loc[df_analysis_message_info[\"couple_id\"] == couple_id, [\"woa\", \"dq\", \"m1_error\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78bc054f-a613-46c3-a66e-2bdb290ba121",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,5))\n",
    "sns.pointplot(x=\"treatment_class\", y=\"previous_exp_pos_msg\", data=df_analysis_message_info.query(\"task_index != 1\"), \n",
    "              label=\"Positive peer messages\", order=[\"NoFeedback_NoExpl\", \"ExplOnly\", \"FeedbackOnly\", \"Feedback+Expl\"], ax=ax, linestyles=\"\")\n",
    "\n",
    "sns.pointplot(x=\"treatment_class\", y=\"previous_exp_neg_msg\", data=df_analysis_message_info.query(\"task_index != 1\"), \n",
    "              color=\"orange\", label=\"Negative peer messages\", order=[\"NoFeedback_NoExpl\", \"ExplOnly\", \"FeedbackOnly\", \"Feedback+Expl\"], ax=ax, linestyles=\"\")\n",
    "\n",
    "offset = transforms.ScaledTranslation(20/72., 0, ax.figure.dpi_scale_trans)\n",
    "trans = ax.lines[0].get_transform()\n",
    "ax.lines[0].set_transform(trans + offset)\n",
    "\n",
    "for i in range(1,5):\n",
    "    trans = ax.lines[i].get_transform()\n",
    "    ax.lines[i].set_transform(trans + offset)\n",
    "\n",
    "offset = transforms.ScaledTranslation(20/72., 0, ax.figure.dpi_scale_trans)\n",
    "trans = ax.collections[0].get_transform()\n",
    "ax.collections[0].set_transform(trans + offset)\n",
    "\n",
    "\n",
    "plt.xlabel(\"Information displayed\", fontsize=20)\n",
    "plt.ylabel(\"# Messages from peers\", fontsize=20)\n",
    "plt.legend(loc=\"upper left\")\n",
    "\n",
    "plt.xticks(ticks = [0,1,2,3], labels=[\"None\", \"Explanations\\nonly\", \"Feedback\\nonly\", \"Explanations\\n& Feedback\"], fontsize=15)\n",
    "plt.tick_params(labelsize=15)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a30d5b43-b3ef-4b59-8dae-743f234f4b8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "smf.mixedlm(\"zscore_woa ~ previous_exp_pos_msg + previous_exp_neg_msg + previous_n_round_messages\", \n",
    "            data = df_analysis_message_info.query(\"task_index != 1\"),\n",
    "            groups=\"playerId\").fit().summary(xname_fe=[\"Intercept\", \"# positive peer messages in prev. round\", \"# negative peer messages in prev. round\", \"# messages in prev. round\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da34dfc6-bb2f-4e46-a46d-5dfb34bd9101",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Figure 5: Feedback-contingent signals in indicator-based social information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19661609-24d1-43d2-95f4-faab3ea504bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.pointplot(x=\"treatment_feedback\", y=\"zscore_woa\", \n",
    "              data=df_final.query(\"treatment_social=='Indicators' and task_index != 1\").assign(previous_signals_to_rely_more = lambda x: x.previous_signals_to_rely_more.astype(int)), hue=\"previous_signals_to_rely_more\", dodge=0.25, linestyles=\"\")\n",
    "plt.xlabel(\"Is feedback enabled?\", fontsize=18)\n",
    "plt.ylabel(\"WoA measured directly\\nafter observing peer signals\", fontsize=18)\n",
    "plt.legend(title=\"Feedback-contingent peer\\nsignals to increase reliance\")\n",
    "plt.tick_params(labelsize=15)\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
